{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "63ad95cf-88e6-432f-bc87-07061e74f87b",
   "metadata": {},
   "source": [
    "# Contextual keywords generation for Financial report document\n",
    "In this Example we will use a Financial report document taken from https://github.com/patronus-ai/financebench/tree/main/pdfs\n",
    "\n",
    "**Steps**:\n",
    "1) Parse financial report document (pdf file) using LlamaParse.\n",
    "2) Split document into chunks. Here we use basic tokens-based chunking with constant chunk_size, but you can use any other method for chunking.\n",
    "3) Generate contextual keywords for each chunk.\n",
    "4) Create questions related to randomly selected chunks since a predefined test set is unavailable.\n",
    "5) Evaluate the method by retrieving the top five most relevant chunks based on cosine similarity between chunk and question embeddings, checking if the correct chunk is included.\n",
    "6) To compare results with raw content (without keywords), create a new index in the \"Create index\" section and modify the document structure by replacing Document(text='#'+x['keywords']+'\\n'+x['content'], .. ) with Document(text=x['content'], .. ).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c9aca84-89dc-4a58-9079-d9fb3ef78197",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install dependencies\n",
    "!pip install tiktoken\n",
    "!pip install torch\n",
    "!pip install transformers\n",
    "!pip install llama_parse\n",
    "!pip install llama_index\n",
    "\n",
    "import random\n",
    "import os\n",
    "import sys\n",
    "import json\n",
    "from llama_parse import LlamaParse\n",
    "from llama_index.core import SimpleDirectoryReader\n",
    "from llama_index.core.schema import Document\n",
    "from config import LLAMAPARSE_API_KEY\n",
    "\n",
    "# Enable nested async loops\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1adeed54-5415-4b7e-83f8-ba92fe9c494b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# download the pdf file into ./data folder \n",
    "!wget -P data/ https://github.com/patronus-ai/financebench/raw/main/pdfs/AMAZON_2015_10K.pdf\n",
    "!mkdir temp\n",
    "\n",
    "# Parse pdf file\n",
    "parser = LlamaParse(\n",
    "    api_key=LLAMAPARSE_API_KEY,\n",
    "    result_type=\"markdown\"  # \"markdown\" and \"text\" are available\n",
    ")\n",
    "file_extractor = {\".pdf\": parser}\n",
    "documents = SimpleDirectoryReader(input_files=['./data/AMAZON_2015_10K.pdf'], file_extractor=file_extractor).load_data()\n",
    "print(\"pdf file pages:\", len(documents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e562350c-ef4a-4657-856e-16e2c5da93cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into chunks (by tokens)\n",
    "from helper import file_get_contents, file_put_contents, generate_contextual_keywords, get_llm_answer, generate_questions_bychunk\n",
    "import tiktoken\n",
    "enc = tiktoken.get_encoding(\"o200k_base\")\n",
    "\n",
    "def split_into_chunks(content, chunk_size):\n",
    "\ta = enc.encode(content)\n",
    "\tleft, chunks = 0, []\n",
    "\twhile left < len(a):\n",
    "\t\tarr = a[left : left+chunk_size]\n",
    "\t\tchunks.append(enc.decode(arr))\n",
    "\t\tleft+=chunk_size\n",
    "\treturn chunks\n",
    "    \n",
    "def generate_chunked_content(chunks):\n",
    "    chunked_content = \"\"\n",
    "    for idx, text in enumerate(chunks):\n",
    "      chunked_content+=f\"### Chunk {idx+1} ###\\n{text}\\n\\n\"\n",
    "    return chunked_content\n",
    "    \n",
    "\n",
    "# Generate contextual keywords\n",
    "path = \"./temp/chunks2.json\"\n",
    "if not os.path.exists(path):\n",
    "    print(\"Generating keywords..\")\n",
    "    document_content, chunks, chunks2 = \"\", [], []\n",
    "    for doc in documents: document_content+=doc.text+\"\\n\"\n",
    "    chunks1 = split_into_chunks(document_content, 400) #400 -- defaulf value\n",
    "    for i, chunk in enumerate(chunks1):\n",
    "        chunks.append(chunk)\n",
    "        if (len(chunks) > 10 or (i==len(chunks1)-1) and len(chunks)>2):\n",
    "            chunked_content = generate_chunked_content(chunks)\n",
    "            keywords = generate_contextual_keywords(chunked_content)        \n",
    "            print(\"page_end:\", i+1, keywords, len(keywords), len(chunks))            \n",
    "            assert len(keywords) >= len(chunks)\n",
    "            for j in range(len(chunks)): chunks2.append( {\"idx\":j, \"keywords\":keywords[j], \"content\":chunks[j]} )\n",
    "            chunks = []\n",
    "    file_put_contents(path, json.dumps(chunks2))\n",
    "else:\n",
    "    chunks2 = json.loads(file_get_contents(path)) #it has content, keywords, idx\n",
    "\n",
    "\n",
    "# Generate questions\n",
    "path = \"./temp/chunks3.json\"\n",
    "if not os.path.exists(path):\n",
    "    print(\"Generating questions..\")\n",
    "    chunks3 = generate_questions_bychunk(chunks2) \n",
    "    file_put_contents(path, json.dumps(chunks3))\n",
    "else:\n",
    "    chunks3 = json.loads(file_get_contents(path)) #it has content, keywords, questions, idx now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0d40bb95-90d5-49d8-8374-8ab0f4c3fdeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "--- Test: What is the fiscal year end date for Adobe Systems Incorporated's annual report filed with the SEC in 2015? idx: 0\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "{'id': '0'}\n",
      "{'id': '4'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: What is the Commission File Number for Adobe Systems Incorporated? idx: 0\n",
      "{'id': '1'}\n",
      "{'id': '0'}\n",
      "{'id': '10'}\n",
      "{'id': '468'}\n",
      "{'id': '397'}\n",
      "\n",
      "\n",
      "--- Test: In which state is Adobe Systems Incorporated incorporated or organized? idx: 0\n",
      "{'id': '10'}\n",
      "{'id': '280'}\n",
      "{'id': '9'}\n",
      "{'id': '1'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: How is Adobe's business organized? idx: 13\n",
      "{'id': '10'}\n",
      "{'id': '280'}\n",
      "{'id': '10'}\n",
      "{'id': '9'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What are Adobe's two strategic growth opportunities? idx: 13\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "{'id': '13'}\n",
      "{'id': '1'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What are the three reportable segments of Adobe's business? idx: 13\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '184'}\n",
      "{'id': '13'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: What is Adobe Digital Publishing Solution used for? idx: 16\n",
      "{'id': '3'}\n",
      "{'id': '9'}\n",
      "{'id': '3'}\n",
      "{'id': '10'}\n",
      "{'id': '16'}\n",
      "\n",
      "\n",
      "--- Test: What are some examples of mobile apps offered by Adobe's Digital Media business? idx: 16\n",
      "{'id': '5'}\n",
      "{'id': '3'}\n",
      "{'id': '16'}\n",
      "{'id': '0'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: What is the main purpose of Adobe's Document Cloud business? idx: 16\n",
      "{'id': '4'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '3'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What is Adobe's strategy to accelerate the adoption of Creative Cloud? idx: 18\n",
      "{'id': '8'}\n",
      "{'id': '10'}\n",
      "{'id': '18'}\n",
      "{'id': '3'}\n",
      "{'id': '174'}\n",
      "\n",
      "\n",
      "--- Test: How does Adobe's subscription model benefit new customers? idx: 18\n",
      "{'id': '9'}\n",
      "{'id': '0'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '18'}\n",
      "\n",
      "\n",
      "--- Test: What is driving the increase in Adobe's recurring revenue over the next several years? idx: 18\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "{'id': '18'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What is Adobe Stock and how is it integrated with Creative Cloud apps? idx: 29\n",
      "{'id': '29'}\n",
      "{'id': '3'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: How do Adobe's mobile apps extend the capabilities of Creative Cloud desktop tools? idx: 29\n",
      "{'id': '6'}\n",
      "{'id': '29'}\n",
      "{'id': '4'}\n",
      "{'id': '8'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What licensing options are available for Creative Cloud, and how can individuals and teams subscribe to the service? idx: 29\n",
      "{'id': '6'}\n",
      "{'id': '174'}\n",
      "{'id': '10'}\n",
      "{'id': '18'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: Who are the main competitors of Adobe in the electronic signatures market? idx: 48\n",
      "{'id': '48'}\n",
      "{'id': '3'}\n",
      "{'id': '5'}\n",
      "{'id': '7'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What companies compete with Adobe Marketing Cloud in the digital marketing space? idx: 48\n",
      "{'id': '7'}\n",
      "{'id': '48'}\n",
      "{'id': '1'}\n",
      "{'id': '3'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How do Adobe's competitors deliver their solutions to customers in the digital marketing market? idx: 48\n",
      "{'id': '48'}\n",
      "{'id': '7'}\n",
      "{'id': '1'}\n",
      "{'id': '6'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What was Michael Dillon's role at Sun Microsystems from April 2006 to January 2010? idx: 67\n",
      "{'id': '67'}\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "{'id': '0'}\n",
      "{'id': '77'}\n",
      "\n",
      "\n",
      "--- Test: Which company did Michael Dillon join in August 2012 as Senior Vice President, General Counsel and Corporate Secretary? idx: 67\n",
      "{'id': '67'}\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "{'id': '0'}\n",
      "{'id': '77'}\n",
      "\n",
      "\n",
      "--- Test: What was Michael Dillon's position at ONI Systems Corp from October 1999 until June 2002? idx: 67\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "{'id': '67'}\n",
      "{'id': '0'}\n",
      "{'id': '77'}\n",
      "\n",
      "\n",
      "--- Test: What positions did Bryan Lamkin hold at Adobe from 1992 to 2006? idx: 72\n",
      "{'id': '5'}\n",
      "{'id': '4'}\n",
      "{'id': '72'}\n",
      "{'id': '3'}\n",
      "{'id': '77'}\n",
      "\n",
      "\n",
      "--- Test: What company did Bryan Lamkin co-found and serve as CEO of from June 2010 to May 2011? idx: 72\n",
      "{'id': '4'}\n",
      "{'id': '5'}\n",
      "{'id': '72'}\n",
      "{'id': '3'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: Who is the Senior Vice President and Chief Marketing Officer at Adobe? idx: 72\n",
      "{'id': '77'}\n",
      "{'id': '8'}\n",
      "{'id': '9'}\n",
      "{'id': '2'}\n",
      "{'id': '72'}\n",
      "\n",
      "\n",
      "--- Test: What was Mr. Parasnis' role at Oracle from January 2012 to November 2013? idx: 77\n",
      "{'id': '10'}\n",
      "{'id': '77'}\n",
      "{'id': '9'}\n",
      "{'id': '3'}\n",
      "{'id': '67'}\n",
      "\n",
      "\n",
      "--- Test: Who was the General Manager of Microsoft Azure AppFabric at Microsoft from April 2009 to December 2011? idx: 77\n",
      "{'id': '10'}\n",
      "{'id': '77'}\n",
      "{'id': '3'}\n",
      "{'id': '67'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: When did Adobe acquire Omniture, Inc.? idx: 77\n",
      "{'id': '77'}\n",
      "{'id': '2'}\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '457'}\n",
      "\n",
      "\n",
      "--- Test: What are the potential risks associated with collecting and managing customer data? idx: 102\n",
      "{'id': '2'}\n",
      "{'id': '0'}\n",
      "{'id': '102'}\n",
      "{'id': '0'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How can hardware or software failures affect the accuracy of collected data? idx: 102\n",
      "{'id': '102'}\n",
      "{'id': '4'}\n",
      "{'id': '2'}\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What are the potential consequences of a security breach or malware attack on data collection systems? idx: 102\n",
      "{'id': '2'}\n",
      "{'id': '102'}\n",
      "{'id': '0'}\n",
      "{'id': '7'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: What are the potential risks associated with integrating acquired companies or technologies? idx: 103\n",
      "{'id': '103'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '106'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: How might the disruption of integrating an acquisition affect a company's business and employees? idx: 103\n",
      "{'id': '103'}\n",
      "{'id': '5'}\n",
      "{'id': '6'}\n",
      "{'id': '106'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: What are some challenges that a company may face when entering new markets through an acquisition? idx: 103\n",
      "{'id': '103'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '7'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What are the potential consequences of not completing an announced acquisition transaction or integrating an acquired business successfully? idx: 106\n",
      "{'id': '103'}\n",
      "{'id': '5'}\n",
      "{'id': '6'}\n",
      "{'id': '106'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: Why may the company incur substantial costs related to intellectual property rights? idx: 106\n",
      "{'id': '8'}\n",
      "{'id': '106'}\n",
      "{'id': '2'}\n",
      "{'id': '5'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: How can intellectual property disputes and litigation impact the company's business operations? idx: 106\n",
      "{'id': '106'}\n",
      "{'id': '8'}\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What are some of the global business risks faced by Adobe? idx: 127\n",
      "{'id': '127'}\n",
      "{'id': '5'}\n",
      "{'id': '3'}\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: How might delays in obtaining export licenses affect Adobe's revenue? idx: 127\n",
      "{'id': '1'}\n",
      "{'id': '127'}\n",
      "{'id': '6'}\n",
      "{'id': '289'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What percentage of Adobe's employees are located outside the United States? idx: 127\n",
      "{'id': '127'}\n",
      "{'id': '0'}\n",
      "{'id': '192'}\n",
      "{'id': '5'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: What are the potential risks associated with Adobe's investment portfolio? idx: 137\n",
      "{'id': '137'}\n",
      "{'id': '3'}\n",
      "{'id': '3'}\n",
      "{'id': '4'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: Does Adobe own or lease most of its principal properties used during fiscal 2015? idx: 137\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '137'}\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: Where are the two locations where Adobe owns both the building and the land? idx: 137\n",
      "{'id': '6'}\n",
      "{'id': '7'}\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: What type of relief are the plaintiffs seeking in the class action lawsuit? idx: 145\n",
      "{'id': '145'}\n",
      "{'id': '3'}\n",
      "{'id': '1'}\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: Which companies are named as defendants in the antitrust litigation case? idx: 145\n",
      "{'id': '3'}\n",
      "{'id': '145'}\n",
      "{'id': '3'}\n",
      "{'id': '1'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What is the time period specified by the court for employees who worked at Apple to be included in the certified class? idx: 145\n",
      "{'id': '145'}\n",
      "{'id': '3'}\n",
      "{'id': '1'}\n",
      "{'id': '5'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: On which market is Adobe's common stock traded? idx: 149\n",
      "{'id': '149'}\n",
      "{'id': '1'}\n",
      "{'id': '2'}\n",
      "{'id': '29'}\n",
      "{'id': '397'}\n",
      "\n",
      "\n",
      "--- Test: What was the highest sales price per share of Adobe's common stock in Fiscal 2015? idx: 149\n",
      "{'id': '149'}\n",
      "{'id': '399'}\n",
      "{'id': '192'}\n",
      "{'id': '3'}\n",
      "{'id': '393'}\n",
      "\n",
      "\n",
      "--- Test: What is the symbol under which Adobe's common stock is traded on the NASDAQ Global Select Market? idx: 149\n",
      "{'id': '149'}\n",
      "{'id': '1'}\n",
      "{'id': '2'}\n",
      "{'id': '5'}\n",
      "{'id': '29'}\n",
      "\n",
      "\n",
      "--- Test: How much authority was granted by the Board of Directors for the new stock repurchase program in January 2015? idx: 152\n",
      "{'id': '152'}\n",
      "{'id': '240'}\n",
      "{'id': '397'}\n",
      "{'id': '8'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What was the prepayment amount provided to a large financial institution as part of the stock repurchase program in September 2015? idx: 152\n",
      "{'id': '8'}\n",
      "{'id': '152'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: When is the stock repurchase program approved by the Board of Directors set to end? idx: 152\n",
      "{'id': '152'}\n",
      "{'id': '397'}\n",
      "{'id': '10'}\n",
      "{'id': '240'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: What were the two companies that Adobe integrated into its reportable segments in fiscal 2013? idx: 158\n",
      "{'id': '184'}\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: Why did Adobe not present pro forma information for its acquisitions in fiscal years 2015, 2014, and 2013? idx: 158\n",
      "{'id': '158'}\n",
      "{'id': '3'}\n",
      "{'id': '6'}\n",
      "{'id': '7'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: What factors does Adobe use as a basis for its assumptions, judgments, and estimates when preparing its Consolidated Financial Statements? idx: 158\n",
      "{'id': '7'}\n",
      "{'id': '5'}\n",
      "{'id': '8'}\n",
      "{'id': '8'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What is Adobe Creative Cloud and what kind of licenses does it offer? idx: 174\n",
      "{'id': '174'}\n",
      "{'id': '8'}\n",
      "{'id': '29'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: How does Adobe Creative Cloud deliver value to its users compared to perpetual licenses? idx: 174\n",
      "{'id': '174'}\n",
      "{'id': '18'}\n",
      "{'id': '10'}\n",
      "{'id': '4'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What are the different ways that customers can acquire Adobe Creative Cloud capabilities? idx: 174\n",
      "{'id': '3'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '8'}\n",
      "{'id': '16'}\n",
      "\n",
      "\n",
      "--- Test: What are the reportable segments of Adobe's business? idx: 184\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '184'}\n",
      "{'id': '9'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: How much subscription revenue did Adobe's Digital Media segment generate in fiscal 2015? idx: 184\n",
      "{'id': '184'}\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What was the percentage change in subscription revenue for Adobe's Digital Marketing segment from fiscal 2013 to fiscal 2014? idx: 184\n",
      "{'id': '191'}\n",
      "{'id': '184'}\n",
      "{'id': '0'}\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What was the percentage growth of Adobe Marketing Cloud revenue in fiscal 2014 compared to fiscal 2013? idx: 191\n",
      "{'id': '191'}\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: Which product contributed to the increase in Digital Marketing revenue in fiscal 2014? idx: 191\n",
      "{'id': '191'}\n",
      "{'id': '2'}\n",
      "{'id': '4'}\n",
      "{'id': '1'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: Why did revenue from Print and Publishing decrease in fiscal 2014 compared to fiscal 2013? idx: 191\n",
      "{'id': '3'}\n",
      "{'id': '8'}\n",
      "{'id': '7'}\n",
      "{'id': '191'}\n",
      "{'id': '184'}\n",
      "\n",
      "\n",
      "--- Test: What percentage of Adobe's total revenue came from the Americas in fiscal year 2015? idx: 192\n",
      "{'id': '192'}\n",
      "{'id': '0'}\n",
      "{'id': '6'}\n",
      "{'id': '0'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: How much revenue did Adobe generate in the APAC region in fiscal year 2013? idx: 192\n",
      "{'id': '192'}\n",
      "{'id': '0'}\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: What was the percentage increase in Adobe's total revenue from fiscal year 2014 to fiscal year 2015? idx: 192\n",
      "{'id': '6'}\n",
      "{'id': '1'}\n",
      "{'id': '191'}\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What was the total revenue impact of foreign currency fluctuations on Adobe in fiscal year 2015? idx: 196\n",
      "{'id': '8'}\n",
      "{'id': '196'}\n",
      "{'id': '6'}\n",
      "{'id': '192'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: How much did Adobe's hedging program offset the revenue decline in EMEA in fiscal year 2015? idx: 196\n",
      "{'id': '8'}\n",
      "{'id': '196'}\n",
      "{'id': '192'}\n",
      "{'id': '3'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: Which currencies strengthened against the U.S. Dollar in fiscal year 2015, causing a decrease in Adobe's revenue? idx: 196\n",
      "{'id': '8'}\n",
      "{'id': '196'}\n",
      "{'id': '6'}\n",
      "{'id': '10'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: What is the significance of unbilled deferred revenue backlog in Adobe's financial statements? idx: 198\n",
      "{'id': '198'}\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '5'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How much was Adobe's unbilled deferred revenue backlog as of November 27, 2015? idx: 198\n",
      "{'id': '198'}\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '1'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What percentage of Adobe's unbilled deferred revenue backlog was not expected to be billed during fiscal 2016? idx: 198\n",
      "{'id': '198'}\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "{'id': '192'}\n",
      "\n",
      "\n",
      "--- Test: What were the main reasons for the increase in Adobe's cost of subscription revenue from Fiscal 2013 to Fiscal 2014? idx: 203\n",
      "{'id': '191'}\n",
      "{'id': '203'}\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: How did Adobe's acquisition of Neolane in Fiscal 2013 impact the company's compensation and related benefits costs? idx: 203\n",
      "{'id': '0'}\n",
      "{'id': '203'}\n",
      "{'id': '392'}\n",
      "{'id': '158'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What factors contributed to the increase in Adobe's royalty costs in Fiscal 2014? idx: 203\n",
      "{'id': '203'}\n",
      "{'id': '7'}\n",
      "{'id': '8'}\n",
      "{'id': '4'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What was the main reason for the increase in amortization expense during fiscal 2015? idx: 214\n",
      "{'id': '214'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '7'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How did the amortization expense change during fiscal 2014 compared to fiscal 2013? idx: 214\n",
      "{'id': '214'}\n",
      "{'id': '8'}\n",
      "{'id': '8'}\n",
      "{'id': '8'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What was the percentage change in interest and other income (expense), net from fiscal 2014 to fiscal 2013? idx: 214\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '214'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What is the estimated range of potential decreases in underlying unrecognized tax benefits? idx: 222\n",
      "{'id': '1'}\n",
      "{'id': '2'}\n",
      "{'id': '7'}\n",
      "{'id': '222'}\n",
      "{'id': '361'}\n",
      "\n",
      "\n",
      "--- Test: How much cash and cash equivalents did the company have as of November 27, 2015? idx: 222\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What is the purpose of reading the data in conjunction with the Consolidated Statements of Cash Flows? idx: 222\n",
      "{'id': '3'}\n",
      "{'id': '7'}\n",
      "{'id': '4'}\n",
      "{'id': '8'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What were the primary reasons for the net cash used for investing activities in fiscal 2014? idx: 228\n",
      "{'id': '7'}\n",
      "{'id': '228'}\n",
      "{'id': '4'}\n",
      "{'id': '5'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How much was spent on acquisitions of Neolane and Behance in fiscal 2013? idx: 228\n",
      "{'id': '309'}\n",
      "{'id': '2'}\n",
      "{'id': '229'}\n",
      "{'id': '158'}\n",
      "{'id': '228'}\n",
      "\n",
      "\n",
      "--- Test: Where were the construction projects located that involved purchases of property and equipment in fiscal 2013? idx: 228\n",
      "{'id': '228'}\n",
      "{'id': '7'}\n",
      "{'id': '1'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: How much did Adobe issue in senior notes in January 2015? idx: 229\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "{'id': '192'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: What was the purpose of using $600 million of the proceeds from the 2025 Notes offering? idx: 229\n",
      "{'id': '10'}\n",
      "{'id': '0'}\n",
      "{'id': '4'}\n",
      "{'id': '0'}\n",
      "{'id': '229'}\n",
      "\n",
      "\n",
      "--- Test: What is the interest rate of the senior notes due February 1, 2025? idx: 229\n",
      "{'id': '0'}\n",
      "{'id': '8'}\n",
      "{'id': '4'}\n",
      "{'id': '1'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: What was the average price of shares repurchased under the plan in Fiscal 2013? idx: 240\n",
      "{'id': '240'}\n",
      "{'id': '7'}\n",
      "{'id': '399'}\n",
      "{'id': '383'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How many shares were repurchased under the structured repurchases plan approved in April 2012 for Fiscal 2014? idx: 240\n",
      "{'id': '240'}\n",
      "{'id': '7'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '152'}\n",
      "\n",
      "\n",
      "--- Test: When was the board approval date for the structured repurchases plan that resulted in 4,849 shares being repurchased? idx: 240\n",
      "{'id': '240'}\n",
      "{'id': '152'}\n",
      "{'id': '7'}\n",
      "{'id': '8'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What is the estimated increase in the fair value of financial hedging instruments if the U.S. Dollar increases by 10%? idx: 248\n",
      "{'id': '248'}\n",
      "{'id': '2'}\n",
      "{'id': '8'}\n",
      "{'id': '1'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: What model is used to measure the hypothetical market value of option contracts in the sensitivity analysis? idx: 248\n",
      "{'id': '248'}\n",
      "{'id': '8'}\n",
      "{'id': '4'}\n",
      "{'id': '5'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: Why doesn't the company use foreign exchange contracts to hedge local currency denominated operating expenses in certain countries? idx: 248\n",
      "{'id': '248'}\n",
      "{'id': '4'}\n",
      "{'id': '3'}\n",
      "{'id': '7'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: What was Adobe Systems Incorporated's net income for the given period? idx: 264\n",
      "{'id': '9'}\n",
      "{'id': '1'}\n",
      "{'id': '4'}\n",
      "{'id': '3'}\n",
      "{'id': '264'}\n",
      "\n",
      "\n",
      "--- Test: What was the basic net income per share for Adobe Systems Incorporated? idx: 264\n",
      "{'id': '264'}\n",
      "{'id': '3'}\n",
      "{'id': '5'}\n",
      "{'id': '4'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: How many shares were used to compute the diluted net income per share for Adobe Systems Incorporated? idx: 264\n",
      "{'id': '5'}\n",
      "{'id': '1'}\n",
      "{'id': '264'}\n",
      "{'id': '6'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What was the net increase/decrease from derivatives designated as hedging instruments in the reported period? idx: 266\n",
      "{'id': '9'}\n",
      "{'id': '7'}\n",
      "{'id': '8'}\n",
      "{'id': '0'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: How much was the total comprehensive income, net of taxes, for Adobe Systems Incorporated? idx: 266\n",
      "{'id': '1'}\n",
      "{'id': '10'}\n",
      "{'id': '393'}\n",
      "{'id': '264'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: What was the amount of unrealized gains/losses on derivative instruments for Adobe Systems Incorporated? idx: 266\n",
      "{'id': '9'}\n",
      "{'id': '9'}\n",
      "{'id': '266'}\n",
      "{'id': '1'}\n",
      "{'id': '385'}\n",
      "\n",
      "\n",
      "--- Test: What is the total stockholders' equity of Adobe Systems Incorporated as of November 30, 2012? idx: 267\n",
      "{'id': '267'}\n",
      "{'id': '393'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '385'}\n",
      "\n",
      "\n",
      "--- Test: How much net income did Adobe Systems Incorporated report in the period ending November 30, 2012? idx: 267\n",
      "{'id': '1'}\n",
      "{'id': '393'}\n",
      "{'id': '9'}\n",
      "{'id': '4'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What is the amount of treasury stock held by Adobe Systems Incorporated as of November 30, 2012? idx: 267\n",
      "{'id': '6'}\n",
      "{'id': '8'}\n",
      "{'id': '5'}\n",
      "{'id': '2'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: What was the net increase (decrease) in cash and cash equivalents for Adobe Systems Incorporated in November 2015? idx: 278\n",
      "{'id': '278'}\n",
      "{'id': '9'}\n",
      "{'id': '0'}\n",
      "{'id': '6'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: How much cash and cash equivalents did Adobe Systems Incorporated have at the beginning of the year in November 2014? idx: 278\n",
      "{'id': '4'}\n",
      "{'id': '0'}\n",
      "{'id': '4'}\n",
      "{'id': '9'}\n",
      "{'id': '278'}\n",
      "\n",
      "\n",
      "--- Test: What was the effect of foreign currency exchange rates on Adobe Systems Incorporated's cash flows in November 2013? idx: 278\n",
      "{'id': '278'}\n",
      "{'id': '8'}\n",
      "{'id': '196'}\n",
      "{'id': '10'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: When was Adobe Systems Incorporated founded? idx: 280\n",
      "{'id': '280'}\n",
      "{'id': '10'}\n",
      "{'id': '9'}\n",
      "{'id': '468'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: How does Adobe Systems Incorporated market and license its products and services? idx: 280\n",
      "{'id': '9'}\n",
      "{'id': '280'}\n",
      "{'id': '9'}\n",
      "{'id': '9'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What models does Adobe Systems Incorporated use to offer its products, besides traditional licensing? idx: 280\n",
      "{'id': '9'}\n",
      "{'id': '9'}\n",
      "{'id': '280'}\n",
      "{'id': '8'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: How does Adobe Systems Incorporated recognize revenue from OEM licensing? idx: 289\n",
      "{'id': '289'}\n",
      "{'id': '0'}\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: What factors does Adobe consider when estimating future returns and rebates for its desktop application product revenue? idx: 289\n",
      "{'id': '289'}\n",
      "{'id': '10'}\n",
      "{'id': '5'}\n",
      "{'id': '4'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: When does Adobe recognize royalty revenue from OEM customers? idx: 289\n",
      "{'id': '289'}\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "{'id': '9'}\n",
      "\n",
      "\n",
      "--- Test: How much was the total final purchase price for Neolane? idx: 309\n",
      "{'id': '309'}\n",
      "{'id': '0'}\n",
      "{'id': '229'}\n",
      "{'id': '158'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What was the allocation of the purchase price for Neolane towards goodwill? idx: 309\n",
      "{'id': '309'}\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "{'id': '4'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: What was the approximate total final purchase price for Behance? idx: 309\n",
      "{'id': '2'}\n",
      "{'id': '309'}\n",
      "{'id': '383'}\n",
      "{'id': '7'}\n",
      "{'id': '399'}\n",
      "\n",
      "\n",
      "--- Test: What is the total value of short-term investments in municipal securities? idx: 315\n",
      "{'id': '5'}\n",
      "{'id': '315'}\n",
      "{'id': '4'}\n",
      "{'id': '3'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How much are the total unrealized gains on short-term investments? idx: 315\n",
      "{'id': '10'}\n",
      "{'id': '5'}\n",
      "{'id': '6'}\n",
      "{'id': '315'}\n",
      "{'id': '316'}\n",
      "\n",
      "\n",
      "--- Test: What is the total value of cash, cash equivalents, and short-term investments? idx: 315\n",
      "{'id': '6'}\n",
      "{'id': '3'}\n",
      "{'id': '4'}\n",
      "{'id': '3'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What was the fair value of corporate bonds and commercial paper as of November 27, 2015? idx: 316\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "{'id': '10'}\n",
      "{'id': '315'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: How much were the gross unrealized losses for municipal securities in 2014? idx: 316\n",
      "{'id': '316'}\n",
      "{'id': '10'}\n",
      "{'id': '9'}\n",
      "{'id': '5'}\n",
      "{'id': '315'}\n",
      "\n",
      "\n",
      "--- Test: What was the fair value of asset-backed securities as of November 28, 2014? idx: 316\n",
      "{'id': '2'}\n",
      "{'id': '324'}\n",
      "{'id': '8'}\n",
      "{'id': '315'}\n",
      "{'id': '316'}\n",
      "\n",
      "\n",
      "--- Test: What is the total value of Adobe Systems Incorporated's debt and marketable equity securities classified as short-term investments? idx: 319\n",
      "{'id': '319'}\n",
      "{'id': '5'}\n",
      "{'id': '137'}\n",
      "{'id': '0'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: How does Adobe Systems Incorporated evaluate whether a security has experienced an other-than-temporary decline in fair value? idx: 319\n",
      "{'id': '319'}\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "{'id': '8'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What is the value of Adobe Systems Incorporated's debt and marketable equity securities that are due within one year? idx: 319\n",
      "{'id': '319'}\n",
      "{'id': '2'}\n",
      "{'id': '0'}\n",
      "{'id': '2'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What is the total value of Adobe Systems Incorporated's liabilities as of November 28, 2014? idx: 324\n",
      "{'id': '0'}\n",
      "{'id': '393'}\n",
      "{'id': '7'}\n",
      "{'id': '385'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: How much of Adobe Systems Incorporated's cash equivalents are invested in money market mutual funds as of November 28, 2014? idx: 324\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '324'}\n",
      "{'id': '4'}\n",
      "{'id': '278'}\n",
      "\n",
      "\n",
      "--- Test: What method is used to determine the fair value of Adobe Systems Incorporated's financial assets and liabilities, as reported on November 28, 2014? idx: 324\n",
      "{'id': '324'}\n",
      "{'id': '319'}\n",
      "{'id': '6'}\n",
      "{'id': '2'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What is the estimated range of potential decreases in underlying unrecognized tax benefits for Adobe Systems Incorporated? idx: 361\n",
      "{'id': '1'}\n",
      "{'id': '7'}\n",
      "{'id': '6'}\n",
      "{'id': '5'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: Why did Adobe Systems Incorporated initiate a restructuring plan in the fourth quarter of fiscal 2014? idx: 361\n",
      "{'id': '362'}\n",
      "{'id': '361'}\n",
      "{'id': '364'}\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: How many positions were reduced as part of Adobe's Fiscal 2014 Restructuring Plan? idx: 361\n",
      "{'id': '362'}\n",
      "{'id': '3'}\n",
      "{'id': '0'}\n",
      "{'id': '361'}\n",
      "{'id': '364'}\n",
      "\n",
      "\n",
      "--- Test: How much did Adobe Systems Incorporated spend on restructuring charges related to its fiscal 2014 restructuring plan? idx: 362\n",
      "{'id': '362'}\n",
      "{'id': '364'}\n",
      "{'id': '0'}\n",
      "{'id': '0'}\n",
      "{'id': '361'}\n",
      "\n",
      "\n",
      "--- Test: What actions did Adobe Systems Incorporated take as part of its fiscal 2014 restructuring plan? idx: 362\n",
      "{'id': '362'}\n",
      "{'id': '361'}\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "{'id': '364'}\n",
      "\n",
      "\n",
      "--- Test: As of November 27, 2015, what was the status of Adobe Systems Incorporated's Other Restructuring Plans? idx: 362\n",
      "{'id': '0'}\n",
      "{'id': '362'}\n",
      "{'id': '364'}\n",
      "{'id': '361'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: How much accrued restructuring charges did Adobe Systems have as of November 27, 2015? idx: 364\n",
      "{'id': '362'}\n",
      "{'id': '0'}\n",
      "{'id': '364'}\n",
      "{'id': '9'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: When does Adobe Systems expect to pay off the majority of its facilities-related liabilities? idx: 364\n",
      "{'id': '7'}\n",
      "{'id': '4'}\n",
      "{'id': '364'}\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: In what year did Adobe adopt its Employee Investment Plan, a retirement savings plan for its U.S. employees? idx: 364\n",
      "{'id': '2'}\n",
      "{'id': '3'}\n",
      "{'id': '6'}\n",
      "{'id': '0'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: What is the expected life of employee stock purchase rights in fiscal years 2015, 2014, and 2013? idx: 376\n",
      "{'id': '376'}\n",
      "{'id': '383'}\n",
      "{'id': '240'}\n",
      "{'id': '8'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What is the range of volatility used to value employee stock purchase rights in fiscal year 2015? idx: 376\n",
      "{'id': '376'}\n",
      "{'id': '383'}\n",
      "{'id': '1'}\n",
      "{'id': '8'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: What was the beginning outstanding balance of restricted stock units in fiscal year 2015? idx: 376\n",
      "{'id': '3'}\n",
      "{'id': '240'}\n",
      "{'id': '4'}\n",
      "{'id': '10'}\n",
      "{'id': '376'}\n",
      "\n",
      "\n",
      "--- Test: How many shares were granted to participants in the 2012 Performance Share Program? idx: 380\n",
      "{'id': '380'}\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '0'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What was the actual performance achievement of participants in the 2012 Performance Share Program as a percentage of target? idx: 380\n",
      "{'id': '380'}\n",
      "{'id': '9'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: How many shares vested in the first quarter of fiscal 2013 under the 2012 Performance Share Program? idx: 380\n",
      "{'id': '380'}\n",
      "{'id': '7'}\n",
      "{'id': '10'}\n",
      "{'id': '8'}\n",
      "{'id': '240'}\n",
      "\n",
      "\n",
      "--- Test: What was the average price at which employees purchased shares under the ESPP during fiscal 2015? idx: 383\n",
      "{'id': '383'}\n",
      "{'id': '376'}\n",
      "{'id': '7'}\n",
      "{'id': '399'}\n",
      "{'id': '240'}\n",
      "\n",
      "\n",
      "--- Test: How many shares were purchased by employees under the ESPP during fiscal 2013? idx: 383\n",
      "{'id': '383'}\n",
      "{'id': '376'}\n",
      "{'id': '6'}\n",
      "{'id': '8'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What is the intrinsic value of shares purchased under the ESPP, and how is it calculated? idx: 383\n",
      "{'id': '8'}\n",
      "{'id': '376'}\n",
      "{'id': '383'}\n",
      "{'id': '6'}\n",
      "{'id': '0'}\n",
      "\n",
      "\n",
      "--- Test: What was the weighted average fair value of options granted by Adobe Systems Incorporated during fiscal 2013? idx: 385\n",
      "{'id': '385'}\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "{'id': '386'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: How much was the total intrinsic value of options exercised by Adobe Systems Incorporated during fiscal 2015? idx: 385\n",
      "{'id': '385'}\n",
      "{'id': '2'}\n",
      "{'id': '386'}\n",
      "{'id': '7'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What is the method used by Adobe Systems Incorporated to calculate the intrinsic value of options exercised? idx: 385\n",
      "{'id': '385'}\n",
      "{'id': '2'}\n",
      "{'id': '386'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "\n",
      "\n",
      "--- Test: What was the weighted average exercise price of Adobe Systems Incorporated's stock options outstanding in fiscal year 2015? idx: 386\n",
      "{'id': '386'}\n",
      "{'id': '385'}\n",
      "{'id': '10'}\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How many shares of stock options were outstanding at Adobe Systems Incorporated as of November 27, 2015? idx: 386\n",
      "{'id': '386'}\n",
      "{'id': '385'}\n",
      "{'id': '2'}\n",
      "{'id': '8'}\n",
      "{'id': '397'}\n",
      "\n",
      "\n",
      "--- Test: What was the aggregate intrinsic value of Adobe Systems Incorporated's stock options outstanding in fiscal year 2014? idx: 386\n",
      "{'id': '385'}\n",
      "{'id': '386'}\n",
      "{'id': '2'}\n",
      "{'id': '8'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What was the total stock-based compensation cost for Adobe Systems Incorporated in fiscal year 2015? idx: 392\n",
      "{'id': '392'}\n",
      "{'id': '393'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: How much did Adobe Systems Incorporated spend on Sales and Marketing in fiscal year 2013? idx: 392\n",
      "{'id': '0'}\n",
      "{'id': '191'}\n",
      "{'id': '10'}\n",
      "{'id': '7'}\n",
      "{'id': '392'}\n",
      "\n",
      "\n",
      "--- Test: What was the cost of Research and Development for Adobe Systems Incorporated in fiscal year 2014? idx: 392\n",
      "{'id': '10'}\n",
      "{'id': '203'}\n",
      "{'id': '7'}\n",
      "{'id': '0'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What were the total stock-based compensation costs for Adobe Systems Incorporated in fiscal year 2015? idx: 393\n",
      "{'id': '393'}\n",
      "{'id': '392'}\n",
      "{'id': '6'}\n",
      "{'id': '4'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: How much tax benefit did Adobe Systems Incorporated record in fiscal year 2014? idx: 393\n",
      "{'id': '0'}\n",
      "{'id': '393'}\n",
      "{'id': '4'}\n",
      "{'id': '7'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What was the total accumulated other comprehensive income (loss) for Adobe Systems Incorporated in fiscal year 2015? idx: 393\n",
      "{'id': '10'}\n",
      "{'id': '0'}\n",
      "{'id': '1'}\n",
      "{'id': '393'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: What is the total amount of authority granted to Adobe Systems Incorporated for stock repurchase through the end of fiscal 2017? idx: 397\n",
      "{'id': '397'}\n",
      "{'id': '399'}\n",
      "{'id': '2'}\n",
      "{'id': '0'}\n",
      "{'id': '152'}\n",
      "\n",
      "\n",
      "--- Test: How much did Adobe Systems Incorporated provide in prepayments for structured stock repurchase agreements during fiscal 2015? idx: 397\n",
      "{'id': '397'}\n",
      "{'id': '399'}\n",
      "{'id': '2'}\n",
      "{'id': '400'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: Under which stock repurchase program was the $425.0 million prepayment made during fiscal 2015? idx: 397\n",
      "{'id': '8'}\n",
      "{'id': '6'}\n",
      "{'id': '152'}\n",
      "{'id': '5'}\n",
      "{'id': '397'}\n",
      "\n",
      "\n",
      "--- Test: How many shares did Adobe Systems Incorporated repurchase during fiscal 2013? idx: 399\n",
      "{'id': '399'}\n",
      "{'id': '397'}\n",
      "{'id': '10'}\n",
      "{'id': '385'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: What was the average price per share of Adobe's stock repurchases during fiscal 2015? idx: 399\n",
      "{'id': '399'}\n",
      "{'id': '240'}\n",
      "{'id': '192'}\n",
      "{'id': '7'}\n",
      "{'id': '397'}\n",
      "\n",
      "\n",
      "--- Test: How were the prepayments for stock repurchases classified on Adobe's Consolidated Balance Sheets? idx: 399\n",
      "{'id': '400'}\n",
      "{'id': '397'}\n",
      "{'id': '2'}\n",
      "{'id': '8'}\n",
      "{'id': '399'}\n",
      "\n",
      "\n",
      "--- Test: How much prepayment did Adobe Systems Incorporated provide to a large financial institution as part of its stock repurchase program? idx: 400\n",
      "{'id': '397'}\n",
      "{'id': '400'}\n",
      "{'id': '2'}\n",
      "{'id': '399'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What is the remaining amount under Adobe's current stock repurchase authority after completing the $150 million stock repurchase agreement? idx: 400\n",
      "{'id': '397'}\n",
      "{'id': '400'}\n",
      "{'id': '399'}\n",
      "{'id': '2'}\n",
      "{'id': '6'}\n",
      "\n",
      "\n",
      "--- Test: How is diluted net income per share calculated by Adobe Systems Incorporated? idx: 400\n",
      "{'id': '264'}\n",
      "{'id': '5'}\n",
      "{'id': '1'}\n",
      "{'id': '6'}\n",
      "{'id': '3'}\n",
      "\n",
      "\n",
      "--- Test: What is the interest rate on the 2015 Notes and the 2020 Notes? idx: 416\n",
      "{'id': '416'}\n",
      "{'id': '0'}\n",
      "{'id': '0'}\n",
      "{'id': '8'}\n",
      "{'id': '8'}\n",
      "\n",
      "\n",
      "--- Test: When are the interest payments due on the Notes? idx: 416\n",
      "{'id': '8'}\n",
      "{'id': '0'}\n",
      "{'id': '0'}\n",
      "{'id': '416'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What is the purpose of the interest rate swaps entered into by the company in June 2014? idx: 416\n",
      "{'id': '1'}\n",
      "{'id': '8'}\n",
      "{'id': '416'}\n",
      "{'id': '2'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What is the name of the plan that governs the Restricted Stock Unit Award Agreements mentioned in the document? idx: 453\n",
      "{'id': '9'}\n",
      "{'id': '10'}\n",
      "{'id': '453'}\n",
      "{'id': '3'}\n",
      "{'id': '5'}\n",
      "\n",
      "\n",
      "--- Test: What is the filing date of the Form 8-K that includes the Director Initial Grant Restricted Stock Unit Award Agreement? idx: 453\n",
      "{'id': '453'}\n",
      "{'id': '9'}\n",
      "{'id': '1'}\n",
      "{'id': '3'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What is the SEC File Number associated with the exhibits listed in the document? idx: 453\n",
      "{'id': '8'}\n",
      "{'id': '4'}\n",
      "{'id': '0'}\n",
      "{'id': '7'}\n",
      "{'id': '453'}\n",
      "\n",
      "\n",
      "--- Test: What is the name of the equity incentive plan adopted by Omniture, Inc. in 1999? idx: 457\n",
      "{'id': '457'}\n",
      "{'id': '7'}\n",
      "{'id': '8'}\n",
      "{'id': '10'}\n",
      "{'id': '1'}\n",
      "\n",
      "\n",
      "--- Test: What is the SEC file number associated with the Omniture 1999 Equity Incentive Plan? idx: 457\n",
      "{'id': '457'}\n",
      "{'id': '7'}\n",
      "{'id': '8'}\n",
      "{'id': '453'}\n",
      "{'id': '460'}\n",
      "\n",
      "\n",
      "--- Test: On what date was the Form S-1 filing made for the Omniture 1999 Plan's Stock Option Agreement? idx: 457\n",
      "{'id': '457'}\n",
      "{'id': '7'}\n",
      "{'id': '9'}\n",
      "{'id': '3'}\n",
      "{'id': '10'}\n",
      "\n",
      "\n",
      "--- Test: What is the name of the company that has an Employee Stock Option Plan mentioned in the document? idx: 460\n",
      "{'id': '6'}\n",
      "{'id': '460'}\n",
      "{'id': '2'}\n",
      "{'id': '5'}\n",
      "{'id': '7'}\n",
      "\n",
      "\n",
      "--- Test: What is the form number of the document that describes the 2012 Director Compensation? idx: 460\n",
      "{'id': '0'}\n",
      "{'id': '3'}\n",
      "{'id': '2'}\n",
      "{'id': '4'}\n",
      "{'id': '2'}\n",
      "\n",
      "\n",
      "--- Test: On what date was the 2012 Executive Annual Incentive Plan filed, according to the document? idx: 460\n",
      "{'id': '0'}\n",
      "{'id': '10'}\n",
      "{'id': '2'}\n",
      "{'id': '460'}\n",
      "{'id': '380'}\n",
      "\n",
      "\n",
      "--- Test: How did Adobe Systems' stock performance compare to the S&P 500 Index over a five-year period? idx: 468\n",
      "{'id': '5'}\n",
      "{'id': '468'}\n",
      "{'id': '399'}\n",
      "{'id': '5'}\n",
      "{'id': '392'}\n",
      "\n",
      "\n",
      "--- Test: What was the cumulative total return of the S&P 500 Software & Services Index in 2015? idx: 468\n",
      "{'id': '5'}\n",
      "{'id': '468'}\n",
      "{'id': '8'}\n",
      "{'id': '2'}\n",
      "{'id': '393'}\n",
      "\n",
      "\n",
      "--- Test: In which year did Adobe Systems' stock price reach approximately $194.85? idx: 468\n",
      "{'id': '399'}\n",
      "{'id': '468'}\n",
      "{'id': '2'}\n",
      "{'id': '149'}\n",
      "{'id': '385'}\n",
      "Test correct, all: 156 180\n"
     ]
    }
   ],
   "source": [
    "# Create Index    \n",
    "from llama_index.core import GPTVectorStoreIndex, StorageContext, load_index_from_storage, Settings\n",
    "from embedding import LocalJinaEmbedding #locally run the jinai embedding model\n",
    "\n",
    "INDEX_DIR = \"./temp/local_index_cache\"\n",
    "if not os.path.exists(INDEX_DIR):\n",
    "    print(\"Creating new index ...\")    \n",
    "    Settings.embed_model = LocalJinaEmbedding()\n",
    "    Settings.llm = None\n",
    "    documents2 = [Document(text='#'+\",\".join(x['keywords'])+'\\n'+x['content'], metadata={\"id\": str(x[\"idx\"])}) for x in chunks3] \n",
    "    index = GPTVectorStoreIndex.from_documents(documents2)\n",
    "    index.storage_context.persist(persist_dir=INDEX_DIR)\n",
    "else:\n",
    "    storage_context = StorageContext.from_defaults(persist_dir=INDEX_DIR)\n",
    "    index = load_index_from_storage(storage_context)\n",
    "query_engine = index.as_query_engine(similarity_top_k=5)\n",
    "\n",
    "# Run tests\n",
    "count, correct = 0, 0\n",
    "for test in chunks3[:]:\n",
    "    if not \"questions\" in test: continue\n",
    "    idx = test[\"idx\"]\n",
    "    for question in test[\"questions\"]:\n",
    "        count+=1\n",
    "        response = query_engine.query(question)\n",
    "        print(\"\\n\\n--- Test:\", question, \"idx:\", idx)\n",
    "        for result in response.source_nodes[:]:\n",
    "            print(result.node.metadata) #prompt+=f\"\\n\\n<Document>\\n                     \n",
    "            if result.node.metadata['id'] == str(idx): correct+=1                \n",
    "\n",
    "print(\"Test correct, all:\", correct, count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4a0eb0b-a47c-4e45-b9f4-299d4ce64193",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
